{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiment: \n",
    "\n",
    "Evaluate pruning by magnitude weighted by coactivations (more thorough evaluation), compare it to baseline (SET).\n",
    "\n",
    "#### Motivation.\n",
    "\n",
    "Check if results are consistently above baseline.\n",
    "\n",
    "#### Conclusion\n",
    "\n",
    "- No significant difference between both models\n",
    "- No support for early stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import tabulate\n",
    "import pprint\n",
    "import click\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ray.tune.commands import *\n",
    "from nupic.research.frameworks.dynamic_sparse.common.browser import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and check data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = ['improved_magpruning_eval1', ]\n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>train_acc_max</th>\n",
       "      <th>train_acc_max_epoch</th>\n",
       "      <th>train_acc_min</th>\n",
       "      <th>train_acc_min_epoch</th>\n",
       "      <th>train_acc_median</th>\n",
       "      <th>train_acc_last</th>\n",
       "      <th>val_acc_max</th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th>val_acc_min</th>\n",
       "      <th>...</th>\n",
       "      <th>momentum</th>\n",
       "      <th>network</th>\n",
       "      <th>num_classes</th>\n",
       "      <th>on_perc</th>\n",
       "      <th>optim_alg</th>\n",
       "      <th>pruning_early_stop</th>\n",
       "      <th>test_noise</th>\n",
       "      <th>use_kwinners</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>weight_prune_perc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_model=DSNNWeightedMag,on_perc=0.2,pruning_ea...</td>\n",
       "      <td>0.999483</td>\n",
       "      <td>93</td>\n",
       "      <td>0.922517</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998808</td>\n",
       "      <td>0.999083</td>\n",
       "      <td>0.9802</td>\n",
       "      <td>35</td>\n",
       "      <td>0.9597</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_model=DSNNMixedHeb,on_perc=0.2,pruning_early...</td>\n",
       "      <td>0.999433</td>\n",
       "      <td>87</td>\n",
       "      <td>0.926283</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998683</td>\n",
       "      <td>0.999333</td>\n",
       "      <td>0.9798</td>\n",
       "      <td>35</td>\n",
       "      <td>0.9603</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2_model=DSNNWeightedMag,on_perc=0.1,pruning_ea...</td>\n",
       "      <td>0.993517</td>\n",
       "      <td>92</td>\n",
       "      <td>0.908733</td>\n",
       "      <td>0</td>\n",
       "      <td>0.990750</td>\n",
       "      <td>0.993150</td>\n",
       "      <td>0.9733</td>\n",
       "      <td>92</td>\n",
       "      <td>0.9506</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_model=DSNNMixedHeb,on_perc=0.1,pruning_early...</td>\n",
       "      <td>0.993217</td>\n",
       "      <td>94</td>\n",
       "      <td>0.905483</td>\n",
       "      <td>0</td>\n",
       "      <td>0.990275</td>\n",
       "      <td>0.993017</td>\n",
       "      <td>0.9725</td>\n",
       "      <td>38</td>\n",
       "      <td>0.9502</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>SGD</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4_model=DSNNWeightedMag,on_perc=0.2,pruning_ea...</td>\n",
       "      <td>0.999400</td>\n",
       "      <td>75</td>\n",
       "      <td>0.927883</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998633</td>\n",
       "      <td>0.999050</td>\n",
       "      <td>0.9818</td>\n",
       "      <td>44</td>\n",
       "      <td>0.9640</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9</td>\n",
       "      <td>MLPHeb</td>\n",
       "      <td>10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>SGD</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Experiment Name  train_acc_max  \\\n",
       "0  0_model=DSNNWeightedMag,on_perc=0.2,pruning_ea...       0.999483   \n",
       "1  1_model=DSNNMixedHeb,on_perc=0.2,pruning_early...       0.999433   \n",
       "2  2_model=DSNNWeightedMag,on_perc=0.1,pruning_ea...       0.993517   \n",
       "3  3_model=DSNNMixedHeb,on_perc=0.1,pruning_early...       0.993217   \n",
       "4  4_model=DSNNWeightedMag,on_perc=0.2,pruning_ea...       0.999400   \n",
       "\n",
       "   train_acc_max_epoch  train_acc_min  train_acc_min_epoch  train_acc_median  \\\n",
       "0                   93       0.922517                    0          0.998808   \n",
       "1                   87       0.926283                    0          0.998683   \n",
       "2                   92       0.908733                    0          0.990750   \n",
       "3                   94       0.905483                    0          0.990275   \n",
       "4                   75       0.927883                    0          0.998633   \n",
       "\n",
       "   train_acc_last  val_acc_max  val_acc_max_epoch  val_acc_min  ...  momentum  \\\n",
       "0        0.999083       0.9802                 35       0.9597  ...       0.9   \n",
       "1        0.999333       0.9798                 35       0.9603  ...       0.9   \n",
       "2        0.993150       0.9733                 92       0.9506  ...       0.9   \n",
       "3        0.993017       0.9725                 38       0.9502  ...       0.9   \n",
       "4        0.999050       0.9818                 44       0.9640  ...       0.9   \n",
       "\n",
       "   network  num_classes  on_perc optim_alg  pruning_early_stop  test_noise  \\\n",
       "0   MLPHeb           10      0.2       SGD                   0       False   \n",
       "1   MLPHeb           10      0.2       SGD                   0       False   \n",
       "2   MLPHeb           10      0.1       SGD                   0       False   \n",
       "3   MLPHeb           10      0.1       SGD                   0       False   \n",
       "4   MLPHeb           10      0.2       SGD                   1       False   \n",
       "\n",
       "   use_kwinners weight_decay weight_prune_perc  \n",
       "0         False       0.0001               NaN  \n",
       "1         False       0.0001               NaN  \n",
       "2         False       0.0001               NaN  \n",
       "3         False       0.0001               NaN  \n",
       "4         False       0.0001               NaN  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace hebbian prine\n",
    "df['hebbian_prune_perc'] = df['hebbian_prune_perc'].replace(np.nan, 0.0, regex=True)\n",
    "df['weight_prune_perc'] = df['weight_prune_perc'].replace(np.nan, 0.0, regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',\n",
       "       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',\n",
       "       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',\n",
       "       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'epochs',\n",
       "       'experiment_file_name', 'trial_time', 'mean_epoch_time', 'batch_norm',\n",
       "       'data_dir', 'dataset_name', 'debug_sparse', 'debug_weights', 'device',\n",
       "       'hebbian_grow', 'hebbian_prune_perc', 'hidden_sizes', 'input_size',\n",
       "       'learning_rate', 'lr_gamma', 'lr_milestones', 'lr_scheduler', 'model',\n",
       "       'momentum', 'network', 'num_classes', 'on_perc', 'optim_alg',\n",
       "       'pruning_early_stop', 'test_noise', 'use_kwinners', 'weight_decay',\n",
       "       'weight_prune_perc'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(288, 42)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Experiment Name         1_model=DSNNMixedHeb,on_perc=0.2,pruning_early...\n",
       "train_acc_max                                                    0.999433\n",
       "train_acc_max_epoch                                                    87\n",
       "train_acc_min                                                    0.926283\n",
       "train_acc_min_epoch                                                     0\n",
       "train_acc_median                                                 0.998683\n",
       "train_acc_last                                                   0.999333\n",
       "val_acc_max                                                        0.9798\n",
       "val_acc_max_epoch                                                      35\n",
       "val_acc_min                                                        0.9603\n",
       "val_acc_min_epoch                                                       0\n",
       "val_acc_median                                                     0.9784\n",
       "val_acc_last                                                       0.9789\n",
       "epochs                                                                100\n",
       "experiment_file_name    /Users/lsouza/nta/results/improved_magpruning_...\n",
       "trial_time                                                        57.1836\n",
       "mean_epoch_time                                                  0.571836\n",
       "batch_norm                                                           True\n",
       "data_dir                                        /home/ubuntu/nta/datasets\n",
       "dataset_name                                                        MNIST\n",
       "debug_sparse                                                         True\n",
       "debug_weights                                                        True\n",
       "device                                                               cuda\n",
       "hebbian_grow                                                        False\n",
       "hebbian_prune_perc                                                      0\n",
       "hidden_sizes                                                          100\n",
       "input_size                                                            784\n",
       "learning_rate                                                         0.1\n",
       "lr_gamma                                                              0.1\n",
       "lr_milestones                                                          60\n",
       "lr_scheduler                                                  MultiStepLR\n",
       "model                                                        DSNNMixedHeb\n",
       "momentum                                                              0.9\n",
       "network                                                            MLPHeb\n",
       "num_classes                                                            10\n",
       "on_perc                                                               0.2\n",
       "optim_alg                                                             SGD\n",
       "pruning_early_stop                                                      0\n",
       "test_noise                                                          False\n",
       "use_kwinners                                                        False\n",
       "weight_decay                                                       0.0001\n",
       "weight_prune_perc                                                       0\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "DSNNMixedHeb       144\n",
       "DSNNWeightedMag    144\n",
       "Name: model, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('model')['model'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiment Details"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "base_exp_config = dict(\n",
    "    device=\"cuda\",\n",
    "    # dataset related\n",
    "    dataset_name=\"CIFAR10\",\n",
    "    input_size=3072,\n",
    "    num_classes=10,\n",
    "    stats_mean=(0.4914, 0.4822, 0.4465),\n",
    "    stats_std=(0.2023, 0.1994, 0.2010),\n",
    "    data_dir=\"~/nta/datasets\",\n",
    "    # model related\n",
    "    model=\"DSNNMixedHeb\",\n",
    "    network=\"MLPHeb\",\n",
    "    init_weights=True,\n",
    "    batch_norm=True,\n",
    "    dropout=False,\n",
    "    kwinners=True,\n",
    "    percent_on=0.3,\n",
    "    boost_strength=1.4,\n",
    "    boost_strength_factor=0.7,\n",
    "    # optimizer related\n",
    "    optim_alg=\"SGD\",\n",
    "    momentum=0.9,\n",
    "    learning_rate=0.01,\n",
    "    weight_decay=1e-4,\n",
    "    # sparse related\n",
    "    epsilon=100,\n",
    "    start_sparse=1,\n",
    "    end_sparse=None,\n",
    "    weight_prune_perc=0.45,\n",
    "    hebbian_prune_perc=0.45,\n",
    "    pruning_es=True,\n",
    "    pruning_es_patience=0,\n",
    "    pruning_es_window_size=5,\n",
    "    pruning_es_threshold=0.02,\n",
    "    pruning_interval=1,\n",
    "    # additional validation\n",
    "    test_noise=True,\n",
    "    noise_level=0.1,\n",
    "    # debugging\n",
    "    debug_weights=True,\n",
    "    debug_sparse=True,\n",
    ")\n",
    "\n",
    "# ray configurations\n",
    "tune_config = dict(\n",
    "    name=\"hebbian-gs-test\",\n",
    "    num_samples=1,\n",
    "    local_dir=os.path.expanduser(\"~/nta/results\"),\n",
    "    checkpoint_freq=0,\n",
    "    checkpoint_at_end=False,\n",
    "    stop={\"training_iteration\": 1000},  # 300 in cifar\n",
    "    resources_per_trial={\"cpu\": 1, \"gpu\": 0.33},\n",
    "    loggers=DEFAULT_LOGGERS,\n",
    "    verbose=1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Did any  trials failed?\n",
    "df[df[\"epochs\"]<30][\"epochs\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(288, 42)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing failed or incomplete trials\n",
    "df_origin = df.copy()\n",
    "df = df_origin[df_origin[\"epochs\"]>=30]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: epochs, dtype: int64)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# which ones failed?\n",
    "# failed, or still ongoing?\n",
    "df_origin['failed'] = df_origin[\"epochs\"]<30\n",
    "df_origin[df_origin['failed']]['epochs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper functions\n",
    "def mean_and_std(s):\n",
    "    return \"{:.3f} ± {:.3f}\".format(s.mean(), s.std())\n",
    "\n",
    "def round_mean(s):\n",
    "    return \"{:.0f}\".format(round(s.mean()))\n",
    "\n",
    "stats = ['min', 'max', 'mean', 'std']\n",
    "\n",
    "def agg(columns, filter=None, round=3):\n",
    "    if filter is None:\n",
    "        return (df.groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n",
    "    else:\n",
    "        return (df[filter].groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Does improved weight pruning outperforms regular SET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>49</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.003</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>48</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.003</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                val_acc_max_epoch val_acc_max                      model\n",
       "                       round_mean         min    max   mean    std count\n",
       "model                                                                   \n",
       "DSNNMixedHeb                   49       0.972  0.986  0.982  0.003   144\n",
       "DSNNWeightedMag                48       0.973  0.986  0.981  0.003   144"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>on_perc</th>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.1</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>46</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>47</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.2</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>53</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.002</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>49</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.001</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        val_acc_max_epoch val_acc_max                       \\\n",
       "                               round_mean         min    max   mean    std   \n",
       "on_perc model                                                                \n",
       "0.1     DSNNMixedHeb                   46       0.972  0.984  0.980  0.003   \n",
       "        DSNNWeightedMag                47       0.973  0.983  0.980  0.003   \n",
       "0.2     DSNNMixedHeb                   53       0.980  0.986  0.983  0.002   \n",
       "        DSNNWeightedMag                49       0.979  0.986  0.983  0.001   \n",
       "\n",
       "                        model  \n",
       "                        count  \n",
       "on_perc model                  \n",
       "0.1     DSNNMixedHeb       72  \n",
       "        DSNNWeightedMag    72  \n",
       "0.2     DSNNMixedHeb       72  \n",
       "        DSNNWeightedMag    72  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['on_perc', 'model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_prune_perc</th>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.0</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>38</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.003</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>42</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.003</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.1</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>48</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.002</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>49</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.002</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.2</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>48</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.002</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>51</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.002</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.3</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>55</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.001</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>52</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.002</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.4</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>53</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.002</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>52</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.002</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.5</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>53</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.002</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>43</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.002</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  val_acc_max_epoch val_acc_max                \\\n",
       "                                         round_mean         min    max   mean   \n",
       "weight_prune_perc model                                                         \n",
       "0.0               DSNNMixedHeb                   38       0.972  0.982  0.978   \n",
       "                  DSNNWeightedMag                42       0.973  0.983  0.978   \n",
       "0.1               DSNNMixedHeb                   48       0.980  0.986  0.982   \n",
       "                  DSNNWeightedMag                49       0.979  0.985  0.983   \n",
       "0.2               DSNNMixedHeb                   48       0.979  0.985  0.982   \n",
       "                  DSNNWeightedMag                51       0.980  0.985  0.982   \n",
       "0.3               DSNNMixedHeb                   55       0.980  0.985  0.983   \n",
       "                  DSNNWeightedMag                52       0.980  0.986  0.983   \n",
       "0.4               DSNNMixedHeb                   53       0.981  0.985  0.983   \n",
       "                  DSNNWeightedMag                52       0.979  0.985  0.982   \n",
       "0.5               DSNNMixedHeb                   53       0.979  0.985  0.982   \n",
       "                  DSNNWeightedMag                43       0.978  0.984  0.981   \n",
       "\n",
       "                                         model  \n",
       "                                     std count  \n",
       "weight_prune_perc model                         \n",
       "0.0               DSNNMixedHeb     0.003    24  \n",
       "                  DSNNWeightedMag  0.003    24  \n",
       "0.1               DSNNMixedHeb     0.002    24  \n",
       "                  DSNNWeightedMag  0.002    24  \n",
       "0.2               DSNNMixedHeb     0.002    24  \n",
       "                  DSNNWeightedMag  0.002    24  \n",
       "0.3               DSNNMixedHeb     0.001    24  \n",
       "                  DSNNWeightedMag  0.002    24  \n",
       "0.4               DSNNMixedHeb     0.002    24  \n",
       "                  DSNNWeightedMag  0.002    24  \n",
       "0.5               DSNNMixedHeb     0.002    24  \n",
       "                  DSNNWeightedMag  0.002    24  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['weight_prune_perc', 'model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg(['on_perc', 'pruning_early_stop', 'model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>on_perc</th>\n",
       "      <th>pruning_early_stop</th>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">0.1</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>47</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>52</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.003</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>43</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>54</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>50</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>38</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>43</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>46</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.002</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">0.2</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>46</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.002</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>48</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.001</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>54</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.002</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>49</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.001</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>53</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.002</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>49</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.002</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
       "      <th>DSNNMixedHeb</th>\n",
       "      <td>57</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.001</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DSNNWeightedMag</th>\n",
       "      <td>52</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.001</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           val_acc_max_epoch val_acc_max  \\\n",
       "                                                  round_mean         min   \n",
       "on_perc pruning_early_stop model                                           \n",
       "0.1     0                  DSNNMixedHeb                   47       0.972   \n",
       "                           DSNNWeightedMag                52       0.973   \n",
       "        1                  DSNNMixedHeb                   43       0.973   \n",
       "                           DSNNWeightedMag                54       0.974   \n",
       "        2                  DSNNMixedHeb                   50       0.974   \n",
       "                           DSNNWeightedMag                38       0.973   \n",
       "        3                  DSNNMixedHeb                   43       0.974   \n",
       "                           DSNNWeightedMag                46       0.975   \n",
       "0.2     0                  DSNNMixedHeb                   46       0.980   \n",
       "                           DSNNWeightedMag                48       0.980   \n",
       "        1                  DSNNMixedHeb                   54       0.980   \n",
       "                           DSNNWeightedMag                49       0.979   \n",
       "        2                  DSNNMixedHeb                   53       0.980   \n",
       "                           DSNNWeightedMag                49       0.979   \n",
       "        3                  DSNNMixedHeb                   57       0.980   \n",
       "                           DSNNWeightedMag                52       0.981   \n",
       "\n",
       "                                                                model  \n",
       "                                              max   mean    std count  \n",
       "on_perc pruning_early_stop model                                       \n",
       "0.1     0                  DSNNMixedHeb     0.983  0.980  0.003    18  \n",
       "                           DSNNWeightedMag  0.982  0.979  0.003    18  \n",
       "        1                  DSNNMixedHeb     0.983  0.980  0.003    18  \n",
       "                           DSNNWeightedMag  0.983  0.980  0.003    18  \n",
       "        2                  DSNNMixedHeb     0.984  0.980  0.003    18  \n",
       "                           DSNNWeightedMag  0.983  0.980  0.003    18  \n",
       "        3                  DSNNMixedHeb     0.982  0.980  0.003    18  \n",
       "                           DSNNWeightedMag  0.983  0.980  0.002    18  \n",
       "0.2     0                  DSNNMixedHeb     0.985  0.983  0.002    18  \n",
       "                           DSNNWeightedMag  0.985  0.983  0.001    18  \n",
       "        1                  DSNNMixedHeb     0.985  0.984  0.002    18  \n",
       "                           DSNNWeightedMag  0.986  0.983  0.001    18  \n",
       "        2                  DSNNMixedHeb     0.986  0.983  0.002    18  \n",
       "                           DSNNWeightedMag  0.985  0.983  0.002    18  \n",
       "        3                  DSNNMixedHeb     0.985  0.984  0.001    18  \n",
       "                           DSNNWeightedMag  0.985  0.983  0.001    18  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['on_perc', 'pruning_early_stop', 'model'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* No significant difference between the two approaches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### What is the impact of early stopping?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pruning_early_stop</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>48</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.003</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.003</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>48</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.003</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>49</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.003</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   val_acc_max_epoch val_acc_max                      model\n",
       "                          round_mean         min    max   mean    std count\n",
       "pruning_early_stop                                                         \n",
       "0                                 48       0.972  0.985  0.982  0.003    72\n",
       "1                                 50       0.973  0.986  0.982  0.003    72\n",
       "2                                 48       0.973  0.986  0.981  0.003    72\n",
       "3                                 49       0.974  0.985  0.981  0.003    72"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['pruning_early_stop'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>pruning_early_stop</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">DSNNMixedHeb</th>\n",
       "      <th>0</th>\n",
       "      <td>46</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>52</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">DSNNWeightedMag</th>\n",
       "      <th>0</th>\n",
       "      <td>50</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>44</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>49</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.981</td>\n",
       "      <td>0.002</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   val_acc_max_epoch val_acc_max         \\\n",
       "                                          round_mean         min    max   \n",
       "model           pruning_early_stop                                        \n",
       "DSNNMixedHeb    0                                 46       0.972  0.985   \n",
       "                1                                 49       0.973  0.985   \n",
       "                2                                 52       0.974  0.986   \n",
       "                3                                 50       0.974  0.985   \n",
       "DSNNWeightedMag 0                                 50       0.973  0.985   \n",
       "                1                                 51       0.974  0.986   \n",
       "                2                                 44       0.973  0.985   \n",
       "                3                                 49       0.975  0.985   \n",
       "\n",
       "                                                 model  \n",
       "                                     mean    std count  \n",
       "model           pruning_early_stop                      \n",
       "DSNNMixedHeb    0                   0.982  0.003    36  \n",
       "                1                   0.982  0.003    36  \n",
       "                2                   0.982  0.003    36  \n",
       "                3                   0.982  0.003    36  \n",
       "DSNNWeightedMag 0                   0.981  0.003    36  \n",
       "                1                   0.982  0.003    36  \n",
       "                2                   0.981  0.003    36  \n",
       "                3                   0.981  0.002    36  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['model', 'pruning_early_stop'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>on_perc</th>\n",
       "      <th>pruning_early_stop</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">0.1</th>\n",
       "      <th>0</th>\n",
       "      <td>49</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>48</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>44</td>\n",
       "      <td>0.973</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.003</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44</td>\n",
       "      <td>0.974</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.002</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">0.2</th>\n",
       "      <th>0</th>\n",
       "      <td>47</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.001</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.001</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>51</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.002</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>54</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.001</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           val_acc_max_epoch val_acc_max                       \\\n",
       "                                  round_mean         min    max   mean    std   \n",
       "on_perc pruning_early_stop                                                      \n",
       "0.1     0                                 49       0.972  0.983  0.980  0.003   \n",
       "        1                                 48       0.973  0.983  0.980  0.003   \n",
       "        2                                 44       0.973  0.984  0.980  0.003   \n",
       "        3                                 44       0.974  0.983  0.980  0.002   \n",
       "0.2     0                                 47       0.980  0.985  0.983  0.001   \n",
       "        1                                 51       0.979  0.986  0.983  0.001   \n",
       "        2                                 51       0.979  0.986  0.983  0.002   \n",
       "        3                                 54       0.980  0.985  0.983  0.001   \n",
       "\n",
       "                           model  \n",
       "                           count  \n",
       "on_perc pruning_early_stop        \n",
       "0.1     0                     36  \n",
       "        1                     36  \n",
       "        2                     36  \n",
       "        3                     36  \n",
       "0.2     0                     36  \n",
       "        1                     36  \n",
       "        2                     36  \n",
       "        3                     36  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['on_perc', 'pruning_early_stop'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Results are not strong enough. But it is has a slight increase in acc (0.1%), not greater than the standard deviation, when pruning stops at the first learning rate decay for WeightedMag model and at the second learning rate decay for DSSNMixedHeb models.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
